#!/bin/bash

#PBS -N quokka_GPU_64nodes
#PBS -P jh2
#PBS -q gpuvolta
#PBS -l ncpus=3072
#PBS -l ngpus=256
#PBS -l mem=24448GB
#PBS -m aeb
#PBS -M ben.wibking@anu.edu.au
#PBS -l wd
#PBS -l walltime=1:00:00
#PBS -l storage=scratch/jh2+gdata/jh2
#PBS -l jobfs=25600GB

# --- initialize Quokka ---
MPI_OPTIONS="-np $PBS_NGPUS --map-by numa:SPAN --bind-to numa --mca pml ucx -x HCOLL_ENABLE_MCAST=0"
echo "Using MPI_OPTIONS = $MPI_OPTIONS"

mpirun $MPI_OPTIONS ./scripts/gpu_wrapper_shell_2048.sh > shell_64nodes.log






